{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Untitled26.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "8e85dccc8c2d4653beba8ecd5d42182a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_a32bb0226932477bbea72d2b362b7943",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_07bacb155210489da8f3d852be7a9782",
              "IPY_MODEL_5f00fe6902174337877e673812950690"
            ]
          }
        },
        "a32bb0226932477bbea72d2b362b7943": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "07bacb155210489da8f3d852be7a9782": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_9981f99736fb4f2e995b0a708580f318",
            "_dom_classes": [],
            "description": "models/electra/generator/pytorch_model.bin: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 260982246,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 260982246,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_56d9f3ddae8d4071a6dc71b637021817"
          }
        },
        "5f00fe6902174337877e673812950690": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_86f9e3e9538f4abf8e260fbb7a344774",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 249M/249M [00:53&lt;00:00, 4.85MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b64eb0b98f1a4fbea29b661cdcdc2441"
          }
        },
        "9981f99736fb4f2e995b0a708580f318": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "56d9f3ddae8d4071a6dc71b637021817": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "86f9e3e9538f4abf8e260fbb7a344774": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b64eb0b98f1a4fbea29b661cdcdc2441": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d72d0af589424021963194b603a4c3a3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_33dcde905c5648fa83ef81ca17e66937",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_2412001d8a554d6ca4d321158caf37a1",
              "IPY_MODEL_c245761ad9184b9d94b3cefbcb2d870a"
            ]
          }
        },
        "33dcde905c5648fa83ef81ca17e66937": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2412001d8a554d6ca4d321158caf37a1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_38e499b1528640c48c5cc33154a4c751",
            "_dom_classes": [],
            "description": "models/electra/discriminator/pytorch_model.bin: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1679849201,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1679849201,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_34d5b8895f744b3e92dea1857bee4ef7"
          }
        },
        "c245761ad9184b9d94b3cefbcb2d870a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_3453dd9ee6a942879339288a0bb5d342",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.56G/1.56G [00:47&lt;00:00, 35.6MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c4b762a2e73a4994a06a54a574a062d2"
          }
        },
        "38e499b1528640c48c5cc33154a4c751": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "34d5b8895f744b3e92dea1857bee4ef7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3453dd9ee6a942879339288a0bb5d342": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c4b762a2e73a4994a06a54a574a062d2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0033a8a7997145768bd896896a60ba9f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d84c0ea534e5408aba5648f2c14e6ee7",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_70ea2e8a23704b2a8798c839db8c9b6c",
              "IPY_MODEL_0cc614bd477641f18d421ff225792ba9"
            ]
          }
        },
        "d84c0ea534e5408aba5648f2c14e6ee7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "70ea2e8a23704b2a8798c839db8c9b6c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_2170495e2b464109bd2e8d40fb672950",
            "_dom_classes": [],
            "description": "models/electra/generator/config.json: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 463,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 463,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ed765545f20c43979ec8349a9095b132"
          }
        },
        "0cc614bd477641f18d421ff225792ba9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_bd9bf523f2264c568cdffcc77a01cfc9",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 463/463 [00:00&lt;00:00, 771B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c07131e063734ac790695bd3232d3de4"
          }
        },
        "2170495e2b464109bd2e8d40fb672950": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ed765545f20c43979ec8349a9095b132": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bd9bf523f2264c568cdffcc77a01cfc9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c07131e063734ac790695bd3232d3de4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "08d1768210cb4fea8e44b7f81ee11e00": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_782593668d31447f840f41f35cb110fe",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b93d21e52d4745c2a7287ba4e0afdff8",
              "IPY_MODEL_252ab8b655574f88adf490f6ba6c0268"
            ]
          }
        },
        "782593668d31447f840f41f35cb110fe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b93d21e52d4745c2a7287ba4e0afdff8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_8884e744d7964710a56554880a20bc80",
            "_dom_classes": [],
            "description": "models/electra/discriminator/config.json: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 468,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 468,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1c4c44dcb07945a7b5156a912aab2f5f"
          }
        },
        "252ab8b655574f88adf490f6ba6c0268": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ea2b55b61b664ee5a175f8eb7dbfe651",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 468/468 [00:08&lt;00:00, 56.3B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_11a46128d73349d38a466a81e174e8f9"
          }
        },
        "8884e744d7964710a56554880a20bc80": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1c4c44dcb07945a7b5156a912aab2f5f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ea2b55b61b664ee5a175f8eb7dbfe651": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "11a46128d73349d38a466a81e174e8f9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "205094bdd14e4fdf8e3d1d748814fded": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_2f5281bf5a7e4b488e27f581dc7d519f",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_12b8b69731394862b1c60181e70c3172",
              "IPY_MODEL_d20ce9960ef4417b825a862068c1d633"
            ]
          }
        },
        "2f5281bf5a7e4b488e27f581dc7d519f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "12b8b69731394862b1c60181e70c3172": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a926a63bbd2b4b638d4f624ac132d254",
            "_dom_classes": [],
            "description": "models/electra/vocab.txt: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 81,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 81,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d3c74e7b572749448703f5dfef58ccf2"
          }
        },
        "d20ce9960ef4417b825a862068c1d633": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_37d096c4249b4443a48db92af790ae1d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 81.0/81.0 [00:01&lt;00:00, 69.0B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_378210403ec1472ba0975707d0eb41b6"
          }
        },
        "a926a63bbd2b4b638d4f624ac132d254": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d3c74e7b572749448703f5dfef58ccf2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "37d096c4249b4443a48db92af790ae1d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "378210403ec1472ba0975707d0eb41b6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/agemagician/Prot-Transformers/blob/master/Benchmark/ProtElectra.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TqnIJfnOV-no",
        "colab_type": "text"
      },
      "source": [
        "<b>Benchmark ProtElectra Model using GPU or CPU</b>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YGknPA-1WJyt",
        "colab_type": "text"
      },
      "source": [
        "<b>1. Load necessry libraries including huggingface transformers</b>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "glTXi_d_V6cy",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        },
        "outputId": "416ac41a-02cc-4c36-d0da-89f4aef3abf8"
      },
      "source": [
        "!pip install -q transformers"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[K     |████████████████████████████████| 675kB 5.0MB/s \n",
            "\u001b[K     |████████████████████████████████| 890kB 23.2MB/s \n",
            "\u001b[K     |████████████████████████████████| 1.1MB 37.7MB/s \n",
            "\u001b[K     |████████████████████████████████| 3.8MB 43.9MB/s \n",
            "\u001b[?25h  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oLipp_xwWOUE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "from transformers import ElectraForMaskedLM, ElectraModel, pipeline\n",
        "import time\n",
        "from datetime import timedelta\n",
        "import os\n",
        "import requests\n",
        "from tqdm.auto import tqdm"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NTN9-KfTWjqs",
        "colab_type": "text"
      },
      "source": [
        "<B>2. Set the url location of ProtElectr and the vocabulary file</b>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GtlaUgNZWhDG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "generatorModelUrl = 'https://www.dropbox.com/s/5x5et5q84y3r01m/pytorch_model.bin?dl=1'\n",
        "discriminatorModelUrl = 'https://www.dropbox.com/s/9ptrgtc8ranf0pa/pytorch_model.bin?dl=1'\n",
        "\n",
        "generatorConfigUrl = 'https://www.dropbox.com/s/9059fvix18i6why/config.json?dl=1'\n",
        "discriminatorConfigUrl = 'https://www.dropbox.com/s/jq568evzexyla0p/config.json?dl=1'\n",
        "\n",
        "vocabUrl = 'https://www.dropbox.com/s/wck3w1q15bc53s0/vocab.txt?dl=1'"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xBgLUQAZWugs",
        "colab_type": "text"
      },
      "source": [
        "<b>3. Download ProtElectra models and vocabulary files</b>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iAG99e_wWr9J",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "downloadFolderPath = 'models/electra/'"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Mn7adbHfW7Uf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "discriminatorFolderPath = os.path.join(downloadFolderPath, 'discriminator')\n",
        "generatorFolderPath = os.path.join(downloadFolderPath, 'generator')\n",
        "\n",
        "discriminatorModelFilePath = os.path.join(discriminatorFolderPath, 'pytorch_model.bin')\n",
        "generatorModelFilePath = os.path.join(generatorFolderPath, 'pytorch_model.bin')\n",
        "\n",
        "discriminatorConfigFilePath = os.path.join(discriminatorFolderPath, 'config.json')\n",
        "generatorConfigFilePath = os.path.join(generatorFolderPath, 'config.json')\n",
        "\n",
        "vocabFilePath = os.path.join(downloadFolderPath, 'vocab.txt')"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oM1oiBC7W9cd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "if not os.path.exists(discriminatorFolderPath):\n",
        "    os.makedirs(discriminatorFolderPath)\n",
        "if not os.path.exists(generatorFolderPath):\n",
        "    os.makedirs(generatorFolderPath)"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3Qhb0Qe2W_nX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def download_file(url, filename):\n",
        "  response = requests.get(url, stream=True)\n",
        "  with tqdm.wrapattr(open(filename, \"wb\"), \"write\", miniters=1,\n",
        "                    total=int(response.headers.get('content-length', 0)),\n",
        "                    desc=filename) as fout:\n",
        "      for chunk in response.iter_content(chunk_size=4096):\n",
        "          fout.write(chunk)"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ij7lJbcqXA4a",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 262,
          "referenced_widgets": [
            "8e85dccc8c2d4653beba8ecd5d42182a",
            "a32bb0226932477bbea72d2b362b7943",
            "07bacb155210489da8f3d852be7a9782",
            "5f00fe6902174337877e673812950690",
            "9981f99736fb4f2e995b0a708580f318",
            "56d9f3ddae8d4071a6dc71b637021817",
            "86f9e3e9538f4abf8e260fbb7a344774",
            "b64eb0b98f1a4fbea29b661cdcdc2441",
            "d72d0af589424021963194b603a4c3a3",
            "33dcde905c5648fa83ef81ca17e66937",
            "2412001d8a554d6ca4d321158caf37a1",
            "c245761ad9184b9d94b3cefbcb2d870a",
            "38e499b1528640c48c5cc33154a4c751",
            "34d5b8895f744b3e92dea1857bee4ef7",
            "3453dd9ee6a942879339288a0bb5d342",
            "c4b762a2e73a4994a06a54a574a062d2",
            "0033a8a7997145768bd896896a60ba9f",
            "d84c0ea534e5408aba5648f2c14e6ee7",
            "70ea2e8a23704b2a8798c839db8c9b6c",
            "0cc614bd477641f18d421ff225792ba9",
            "2170495e2b464109bd2e8d40fb672950",
            "ed765545f20c43979ec8349a9095b132",
            "bd9bf523f2264c568cdffcc77a01cfc9",
            "c07131e063734ac790695bd3232d3de4",
            "08d1768210cb4fea8e44b7f81ee11e00",
            "782593668d31447f840f41f35cb110fe",
            "b93d21e52d4745c2a7287ba4e0afdff8",
            "252ab8b655574f88adf490f6ba6c0268",
            "8884e744d7964710a56554880a20bc80",
            "1c4c44dcb07945a7b5156a912aab2f5f",
            "ea2b55b61b664ee5a175f8eb7dbfe651",
            "11a46128d73349d38a466a81e174e8f9",
            "205094bdd14e4fdf8e3d1d748814fded",
            "2f5281bf5a7e4b488e27f581dc7d519f",
            "12b8b69731394862b1c60181e70c3172",
            "d20ce9960ef4417b825a862068c1d633",
            "a926a63bbd2b4b638d4f624ac132d254",
            "d3c74e7b572749448703f5dfef58ccf2",
            "37d096c4249b4443a48db92af790ae1d",
            "378210403ec1472ba0975707d0eb41b6"
          ]
        },
        "outputId": "5e461d42-c9ed-4948-eb45-a38e4cfddd89"
      },
      "source": [
        "if not os.path.exists(generatorModelFilePath):\n",
        "    download_file(generatorModelUrl, generatorModelFilePath)\n",
        "\n",
        "if not os.path.exists(discriminatorModelFilePath):\n",
        "    download_file(discriminatorModelUrl, discriminatorModelFilePath)\n",
        "    \n",
        "if not os.path.exists(generatorConfigFilePath):\n",
        "    download_file(generatorConfigUrl, generatorConfigFilePath)\n",
        "\n",
        "if not os.path.exists(discriminatorConfigFilePath):\n",
        "    download_file(discriminatorConfigUrl, discriminatorConfigFilePath)\n",
        "    \n",
        "if not os.path.exists(vocabFilePath):\n",
        "    download_file(vocabUrl, vocabFilePath)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8e85dccc8c2d4653beba8ecd5d42182a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='models/electra/generator/pytorch_model.bin', max=26098224…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d72d0af589424021963194b603a4c3a3",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='models/electra/discriminator/pytorch_model.bin', max=1679…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0033a8a7997145768bd896896a60ba9f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='models/electra/generator/config.json', max=463.0, style=P…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "08d1768210cb4fea8e44b7f81ee11e00",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='models/electra/discriminator/config.json', max=468.0, sty…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "205094bdd14e4fdf8e3d1d748814fded",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='models/electra/vocab.txt', max=81.0, style=ProgressStyle(…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LI7hIlvvXRox",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "<b>4. Load Electra and generator Models</b>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8jKNyeyyXdLh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "electra = ElectraModel.from_pretrained(discriminatorFolderPath)"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DYrkBKDIXCXd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "generator = ElectraForMaskedLM.from_pretrained(generatorFolderPath)"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4YEqregSXpAI",
        "colab_type": "text"
      },
      "source": [
        "<b>5. Load the models into the GPU if avilabile</b>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9AuzukrkXh2n",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xZRVZR9lXwEx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "generator = generator.to(device)\n",
        "generator = generator.eval()"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GJ5-asiTX49b",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "electra = electra.to(device)\n",
        "electra = electra.eval()"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ajffrI3yX-O5",
        "colab_type": "text"
      },
      "source": [
        "<b>6. Benchmark Configuration</b>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c56K4HYUX6Y4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "min_batch_size = 8\n",
        "max_batch_size = 32\n",
        "inc_batch_size = 8\n",
        "\n",
        "min_sequence_length = 64\n",
        "max_sequence_length = 512\n",
        "inc_sequence_length = 64\n",
        "\n",
        "iterations = 10"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4cePeIxrYFnx",
        "colab_type": "text"
      },
      "source": [
        "<b>7. Start Benchmarking Electra</b>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0Pakn8fDYDS-",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 748
        },
        "outputId": "01468c33-4043-4f69-f847-a3fc5c8ffff8"
      },
      "source": [
        "device_name = torch.cuda.get_device_name(device.index) if device.type == 'cuda' else 'CPU'\n",
        "\n",
        "with torch.no_grad():\n",
        "    print((' Benchmarking using ' + device_name + ' ').center(80, '*'))\n",
        "    print(' Start '.center(80, '*'))\n",
        "    for sequence_length in range(min_sequence_length,max_sequence_length+1,inc_sequence_length):\n",
        "        for batch_size in range(min_batch_size,max_batch_size+1,inc_batch_size):\n",
        "            start = time.time()\n",
        "            for i in range(iterations):\n",
        "                input_ids = torch.randint(1, 20, (batch_size,sequence_length)).to(device)\n",
        "                results = electra(input_ids)[0].cpu().numpy()\n",
        "            end = time.time()\n",
        "            ms_per_protein = (end-start)/(iterations*batch_size)\n",
        "            print('Sequence Length: %4d \\t Batch Size: %4d \\t Ms per protein %4.2f' %(sequence_length,batch_size,ms_per_protein))\n",
        "        print(' Done '.center(80, '*'))\n",
        "    print(' Finished '.center(80, '*'))"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "******************* Benchmarking using Tesla P100-PCIE-16GB ********************\n",
            "************************************ Start *************************************\n",
            "Sequence Length:   64 \t Batch Size:    8 \t Ms per protein 0.01\n",
            "Sequence Length:   64 \t Batch Size:   16 \t Ms per protein 0.01\n",
            "Sequence Length:   64 \t Batch Size:   24 \t Ms per protein 0.01\n",
            "Sequence Length:   64 \t Batch Size:   32 \t Ms per protein 0.01\n",
            "************************************* Done *************************************\n",
            "Sequence Length:  128 \t Batch Size:    8 \t Ms per protein 0.02\n",
            "Sequence Length:  128 \t Batch Size:   16 \t Ms per protein 0.02\n",
            "Sequence Length:  128 \t Batch Size:   24 \t Ms per protein 0.02\n",
            "Sequence Length:  128 \t Batch Size:   32 \t Ms per protein 0.02\n",
            "************************************* Done *************************************\n",
            "Sequence Length:  192 \t Batch Size:    8 \t Ms per protein 0.02\n",
            "Sequence Length:  192 \t Batch Size:   16 \t Ms per protein 0.02\n",
            "Sequence Length:  192 \t Batch Size:   24 \t Ms per protein 0.02\n",
            "Sequence Length:  192 \t Batch Size:   32 \t Ms per protein 0.02\n",
            "************************************* Done *************************************\n",
            "Sequence Length:  256 \t Batch Size:    8 \t Ms per protein 0.04\n",
            "Sequence Length:  256 \t Batch Size:   16 \t Ms per protein 0.03\n",
            "Sequence Length:  256 \t Batch Size:   24 \t Ms per protein 0.03\n",
            "Sequence Length:  256 \t Batch Size:   32 \t Ms per protein 0.03\n",
            "************************************* Done *************************************\n",
            "Sequence Length:  320 \t Batch Size:    8 \t Ms per protein 0.04\n",
            "Sequence Length:  320 \t Batch Size:   16 \t Ms per protein 0.04\n",
            "Sequence Length:  320 \t Batch Size:   24 \t Ms per protein 0.04\n",
            "Sequence Length:  320 \t Batch Size:   32 \t Ms per protein 0.04\n",
            "************************************* Done *************************************\n",
            "Sequence Length:  384 \t Batch Size:    8 \t Ms per protein 0.05\n",
            "Sequence Length:  384 \t Batch Size:   16 \t Ms per protein 0.05\n",
            "Sequence Length:  384 \t Batch Size:   24 \t Ms per protein 0.05\n",
            "Sequence Length:  384 \t Batch Size:   32 \t Ms per protein 0.05\n",
            "************************************* Done *************************************\n",
            "Sequence Length:  448 \t Batch Size:    8 \t Ms per protein 0.06\n",
            "Sequence Length:  448 \t Batch Size:   16 \t Ms per protein 0.06\n",
            "Sequence Length:  448 \t Batch Size:   24 \t Ms per protein 0.06\n",
            "Sequence Length:  448 \t Batch Size:   32 \t Ms per protein 0.06\n",
            "************************************* Done *************************************\n",
            "Sequence Length:  512 \t Batch Size:    8 \t Ms per protein 0.07\n",
            "Sequence Length:  512 \t Batch Size:   16 \t Ms per protein 0.07\n",
            "Sequence Length:  512 \t Batch Size:   24 \t Ms per protein 0.06\n",
            "Sequence Length:  512 \t Batch Size:   32 \t Ms per protein 0.07\n",
            "************************************* Done *************************************\n",
            "*********************************** Finished ***********************************\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UM1BAaTHYUcy",
        "colab_type": "text"
      },
      "source": [
        "<b>7. Start Benchmarking Generator</b>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pbfopp5RYRrp",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 748
        },
        "outputId": "9d80e047-0067-4634-b537-177d76f2a9e8"
      },
      "source": [
        "device_name = torch.cuda.get_device_name(device.index) if device.type == 'cuda' else 'CPU'\n",
        "\n",
        "with torch.no_grad():\n",
        "    print((' Benchmarking using ' + device_name + ' ').center(80, '*'))\n",
        "    print(' Start '.center(80, '*'))\n",
        "    for sequence_length in range(min_sequence_length,max_sequence_length+1,inc_sequence_length):\n",
        "        for batch_size in range(min_batch_size,max_batch_size+1,inc_batch_size):\n",
        "            start = time.time()\n",
        "            for i in range(iterations):\n",
        "                input_ids = torch.randint(1, 20, (batch_size,sequence_length)).to(device)\n",
        "                results = generator(input_ids)[0].cpu().numpy()\n",
        "            end = time.time()\n",
        "            ms_per_protein = (end-start)/(iterations*batch_size)\n",
        "            print('Sequence Length: %4d \\t Batch Size: %4d \\t Ms per protein %4.2f' %(sequence_length,batch_size,ms_per_protein))\n",
        "        print(' Done '.center(80, '*'))\n",
        "    print(' Finished '.center(80, '*'))"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "******************* Benchmarking using Tesla P100-PCIE-16GB ********************\n",
            "************************************ Start *************************************\n",
            "Sequence Length:   64 \t Batch Size:    8 \t Ms per protein 0.00\n",
            "Sequence Length:   64 \t Batch Size:   16 \t Ms per protein 0.00\n",
            "Sequence Length:   64 \t Batch Size:   24 \t Ms per protein 0.00\n",
            "Sequence Length:   64 \t Batch Size:   32 \t Ms per protein 0.00\n",
            "************************************* Done *************************************\n",
            "Sequence Length:  128 \t Batch Size:    8 \t Ms per protein 0.00\n",
            "Sequence Length:  128 \t Batch Size:   16 \t Ms per protein 0.00\n",
            "Sequence Length:  128 \t Batch Size:   24 \t Ms per protein 0.00\n",
            "Sequence Length:  128 \t Batch Size:   32 \t Ms per protein 0.00\n",
            "************************************* Done *************************************\n",
            "Sequence Length:  192 \t Batch Size:    8 \t Ms per protein 0.00\n",
            "Sequence Length:  192 \t Batch Size:   16 \t Ms per protein 0.00\n",
            "Sequence Length:  192 \t Batch Size:   24 \t Ms per protein 0.00\n",
            "Sequence Length:  192 \t Batch Size:   32 \t Ms per protein 0.00\n",
            "************************************* Done *************************************\n",
            "Sequence Length:  256 \t Batch Size:    8 \t Ms per protein 0.00\n",
            "Sequence Length:  256 \t Batch Size:   16 \t Ms per protein 0.00\n",
            "Sequence Length:  256 \t Batch Size:   24 \t Ms per protein 0.00\n",
            "Sequence Length:  256 \t Batch Size:   32 \t Ms per protein 0.00\n",
            "************************************* Done *************************************\n",
            "Sequence Length:  320 \t Batch Size:    8 \t Ms per protein 0.01\n",
            "Sequence Length:  320 \t Batch Size:   16 \t Ms per protein 0.01\n",
            "Sequence Length:  320 \t Batch Size:   24 \t Ms per protein 0.01\n",
            "Sequence Length:  320 \t Batch Size:   32 \t Ms per protein 0.00\n",
            "************************************* Done *************************************\n",
            "Sequence Length:  384 \t Batch Size:    8 \t Ms per protein 0.01\n",
            "Sequence Length:  384 \t Batch Size:   16 \t Ms per protein 0.01\n",
            "Sequence Length:  384 \t Batch Size:   24 \t Ms per protein 0.01\n",
            "Sequence Length:  384 \t Batch Size:   32 \t Ms per protein 0.01\n",
            "************************************* Done *************************************\n",
            "Sequence Length:  448 \t Batch Size:    8 \t Ms per protein 0.01\n",
            "Sequence Length:  448 \t Batch Size:   16 \t Ms per protein 0.01\n",
            "Sequence Length:  448 \t Batch Size:   24 \t Ms per protein 0.01\n",
            "Sequence Length:  448 \t Batch Size:   32 \t Ms per protein 0.01\n",
            "************************************* Done *************************************\n",
            "Sequence Length:  512 \t Batch Size:    8 \t Ms per protein 0.01\n",
            "Sequence Length:  512 \t Batch Size:   16 \t Ms per protein 0.01\n",
            "Sequence Length:  512 \t Batch Size:   24 \t Ms per protein 0.01\n",
            "Sequence Length:  512 \t Batch Size:   32 \t Ms per protein 0.01\n",
            "************************************* Done *************************************\n",
            "*********************************** Finished ***********************************\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}